Setup

library(tidyverse)
library(tools)
library(caret)
library(ROCR)
library(caret)
compute_auroc <- function(predictions, labels, title) {
    pred <- prediction(predictions, labels)
    auroc <- performance(pred, measure = 'auc')
    auroc <- auroc@y.values[[1]]
    perf <- performance(pred, 'tpr', 'fpr')
    plot(perf,
         main = title,
         xlim=c(0,1),
         ylim=c(0,1),
         colorize = FALSE)
    abline(a = 0,
           b = 1,
           col = 'red',
           lty = 2)
    text(0.8,
         0.25,
         paste('AUROC:', sprintf('%.3f', round(auroc, 3))))
}
compute_aupr <- function(predictions, labels, title) {
    n_black <- length(labels[labels == 1])
    n_total <- length(labels)
    pred <- prediction(predictions, labels)
    aupr <- performance(pred, measure = 'aucpr')
    aupr <- aupr@y.values[[1]]
    perf <- performance(pred, 'prec', 'rec')
    plot(perf,
         main = title,
         xlim=c(0,1),
         ylim=c(0,1),
         colorize = FALSE)
    abline(a = n_black/n_total,
           b = 0,
           col = 'red',
           lty = 2)
    text(0.2,
         0.2,
         paste('Null AUPR:', sprintf('%.3f', round(n_black / n_total, 3))))
    text(0.8,
         0.2,
         paste('AUPR:', sprintf('%.3f', round(aupr, 3))))
}
confusion_matrix <- function(df, column, threshold=0.5) {
    preds <- as.factor(if_else(df[column] >= threshold, 1, 0))
    confusionMatrix(preds, df$label, positive = '1')
}

Load data

test_data <- read_csv('./out/datasets/test_data.csv') %>%
    select(subject_id, image_id)

── Column specification ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  subject_id = col_character(),
  race = col_character(),
  variable = col_character(),
  value = col_character(),
  image_id = col_double(),
  fundus_location = col_character(),
  segmentation_location = col_character()
)
image_level <- read_csv('./out/probabilities/retcam_filtered_0.csv', col_types = cols()) %>%
    mutate(label = as.factor(if_else(str_detect(img_loc, 'black'), 1, 0)),
           image_id = as.numeric(file_path_sans_ext(basename(img_loc)))) %>%
    select(img_loc, image_id, label, retcam = probability) %>%
    bind_cols(select(read_csv('./out/probabilities/retcam_filtered_0_random.csv', col_types = cols()), retcam_random = probability)) %>%
    
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_random.csv', col_types = cols()), filter_0_random = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0.csv', col_types = cols()), filter_0 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_binarized.csv', col_types = cols()), filter_0_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_skeletonized.csv', col_types = cols()), filter_0_skeletonized = probability)) %>%
    
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_50.csv', col_types = cols()), filter_50 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_50_binarized.csv', col_types = cols()), filter_50_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_50_skeletonized.csv', col_types = cols()), filter_50_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_100.csv', col_types = cols()), filter_100 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_100_binarized.csv', col_types = cols()), filter_100_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_100_skeletonized.csv', col_types = cols()), filter_100_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_150.csv', col_types = cols()), filter_150 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_150_binarized.csv', col_types = cols()), filter_150_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_150_skeletonized.csv', col_types = cols()), filter_150_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_200.csv', col_types = cols()), filter_200 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_200_binarized.csv', col_types = cols()), filter_200_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_200_skeletonized.csv', col_types = cols()), filter_200_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_210.csv', col_types = cols()), filter_210 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_210_binarized.csv', col_types = cols()), filter_210_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_210_skeletonized.csv', col_types = cols()), filter_210_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_220.csv', col_types = cols()), filter_220 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_220_binarized.csv', col_types = cols()), filter_220_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_220_skeletonized.csv', col_types = cols()), filter_220_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_230.csv', col_types = cols()), filter_230 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_230_binarized.csv', col_types = cols()), filter_230_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_230_skeletonized.csv', col_types = cols()), filter_230_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_240.csv', col_types = cols()), filter_240 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_240_binarized.csv', col_types = cols()), filter_240_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_240_skeletonized.csv', col_types = cols()), filter_240_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_250.csv', col_types = cols()), filter_250 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_250_binarized.csv', col_types = cols()), filter_250_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_250_skeletonized.csv', col_types = cols()), filter_250_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_257.csv', col_types = cols()), filter_257 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_257_binarized.csv', col_types = cols()), filter_257_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_257_skeletonized.csv', col_types = cols()), filter_257_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_10.csv', col_types = cols()), filter_10 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_10_binarized.csv', col_types = cols()), filter_10_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_10_skeletonized.csv', col_types = cols()), filter_10_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_75_150.csv', col_types = cols()), filter_75 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_75_150_binarized.csv', col_types = cols()), filter_75_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_75_150_skeletonized.csv', col_types = cols()), filter_75_skeletonized = probability)) %>%
    
    inner_join(test_data, by = 'image_id') %>%
    select(subject_id, everything(), -img_loc)



subject_level <- image_level %>%
    group_by(subject_id) %>%
    mutate(across(c(-image_id, -label), median)) %>%
    ungroup() %>%
    distinct(subject_id, .keep_all = TRUE)

Image-level Analysis

RetCam Images

compute_aupr(image_level$retcam, image_level$label, 'PR: Raw RetCam Images')

compute_auroc(image_level$retcam, image_level$label, 'ROC: Raw RetCam Images')

confusion_matrix(image_level, 'retcam')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 819  10
         1   8 513
                                         
               Accuracy : 0.9867         
                 95% CI : (0.979, 0.9921)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : <2e-16         
                                         
                  Kappa : 0.9719         
                                         
 Mcnemar's Test P-Value : 0.8137         
                                         
            Sensitivity : 0.9809         
            Specificity : 0.9903         
         Pos Pred Value : 0.9846         
         Neg Pred Value : 0.9879         
             Prevalence : 0.3874         
         Detection Rate : 0.3800         
   Detection Prevalence : 0.3859         
      Balanced Accuracy : 0.9856         
                                         
       'Positive' Class : 1              
                                         
compute_aupr(image_level$retcam_random, image_level$label, 'PR: Raw RetCam Images - Shuffled Labels')

compute_auroc(image_level$retcam_random, image_level$label, 'ROC: Raw RetCam Images - Shuffled Labels')

confusion_matrix(image_level, 'retcam_random')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 444 113
         1 383 410
                                          
               Accuracy : 0.6326          
                 95% CI : (0.6062, 0.6584)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 0.06902         
                                          
                  Kappa : 0.293           
                                          
 Mcnemar's Test P-Value : < 2e-16         
                                          
            Sensitivity : 0.7839          
            Specificity : 0.5369          
         Pos Pred Value : 0.5170          
         Neg Pred Value : 0.7971          
             Prevalence : 0.3874          
         Detection Rate : 0.3037          
   Detection Prevalence : 0.5874          
      Balanced Accuracy : 0.6604          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 0

compute_aupr(image_level$filter_0, image_level$label, 'PR: Segmentations: zero all pixels < 0')

compute_auroc(image_level$filter_0, image_level$label, 'ROC: Segmentations: zero all pixels < 0')

confusion_matrix(image_level, 'filter_0')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 758  74
         1  69 449
                                        
               Accuracy : 0.8941        
                 95% CI : (0.8764, 0.91)
    No Information Rate : 0.6126        
    P-Value [Acc > NIR] : <2e-16        
                                        
                  Kappa : 0.7764        
                                        
 Mcnemar's Test P-Value : 0.738         
                                        
            Sensitivity : 0.8585        
            Specificity : 0.9166        
         Pos Pred Value : 0.8668        
         Neg Pred Value : 0.9111        
             Prevalence : 0.3874        
         Detection Rate : 0.3326        
   Detection Prevalence : 0.3837        
      Balanced Accuracy : 0.8875        
                                        
       'Positive' Class : 1             
                                        
compute_aupr(image_level$filter_0_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 0 and binarize')

compute_auroc(image_level$filter_0_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 0 and binarize')

confusion_matrix(image_level, 'filter_0_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 733  31
         1  94 492
                                          
               Accuracy : 0.9074          
                 95% CI : (0.8907, 0.9223)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.8091          
                                          
 Mcnemar's Test P-Value : 2.932e-08       
                                          
            Sensitivity : 0.9407          
            Specificity : 0.8863          
         Pos Pred Value : 0.8396          
         Neg Pred Value : 0.9594          
             Prevalence : 0.3874          
         Detection Rate : 0.3644          
   Detection Prevalence : 0.4341          
      Balanced Accuracy : 0.9135          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_0_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 0 and skeletonize')

compute_auroc(image_level$filter_0_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 0 and skeletonize')

confusion_matrix(image_level, 'filter_0_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 691  38
         1 136 485
                                          
               Accuracy : 0.8711          
                 95% CI : (0.8521, 0.8885)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.7375          
                                          
 Mcnemar's Test P-Value : 1.93e-13        
                                          
            Sensitivity : 0.9273          
            Specificity : 0.8356          
         Pos Pred Value : 0.7810          
         Neg Pred Value : 0.9479          
             Prevalence : 0.3874          
         Detection Rate : 0.3593          
   Detection Prevalence : 0.4600          
      Balanced Accuracy : 0.8814          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_0_random, image_level$label, 'PR: Segmentations: zero all pixels < 0 - Shuffled Labels')

compute_auroc(image_level$filter_0_random, image_level$label, 'ROC: Segmentations: zero all pixels < 0 - Shuffled Labels')

confusion_matrix(image_level, 'filter_0_random')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 703 418
         1 124 105
                                          
               Accuracy : 0.5985          
                 95% CI : (0.5718, 0.6248)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 0.8619          
                                          
                  Kappa : 0.0567          
                                          
 Mcnemar's Test P-Value : <2e-16          
                                          
            Sensitivity : 0.20076         
            Specificity : 0.85006         
         Pos Pred Value : 0.45852         
         Neg Pred Value : 0.62712         
             Prevalence : 0.38741         
         Detection Rate : 0.07778         
   Detection Prevalence : 0.16963         
      Balanced Accuracy : 0.52541         
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 50

compute_aupr(image_level$filter_50, image_level$label, 'PR: Segmentations: zero all pixels < 50')

compute_auroc(image_level$filter_50, image_level$label, 'ROC: Segmentations: zero all pixels < 50')

confusion_matrix(image_level, 'filter_50')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 588  63
         1 239 460
                                          
               Accuracy : 0.7763          
                 95% CI : (0.7531, 0.7983)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.5561          
                                          
 Mcnemar's Test P-Value : < 2.2e-16       
                                          
            Sensitivity : 0.8795          
            Specificity : 0.7110          
         Pos Pred Value : 0.6581          
         Neg Pred Value : 0.9032          
             Prevalence : 0.3874          
         Detection Rate : 0.3407          
   Detection Prevalence : 0.5178          
      Balanced Accuracy : 0.7953          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_50_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 50 and binarize')

compute_auroc(image_level$filter_50_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 50 and binarize')

confusion_matrix(image_level, 'filter_0_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 733  31
         1  94 492
                                          
               Accuracy : 0.9074          
                 95% CI : (0.8907, 0.9223)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.8091          
                                          
 Mcnemar's Test P-Value : 2.932e-08       
                                          
            Sensitivity : 0.9407          
            Specificity : 0.8863          
         Pos Pred Value : 0.8396          
         Neg Pred Value : 0.9594          
             Prevalence : 0.3874          
         Detection Rate : 0.3644          
   Detection Prevalence : 0.4341          
      Balanced Accuracy : 0.9135          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_50_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 50 and skeletonize')

compute_auroc(image_level$filter_50_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 50 and skeletonize')

confusion_matrix(image_level, 'filter_50_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 408  20
         1 419 503
                                          
               Accuracy : 0.6748          
                 95% CI : (0.6491, 0.6998)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 1.202e-06       
                                          
                  Kappa : 0.3991          
                                          
 Mcnemar's Test P-Value : < 2.2e-16       
                                          
            Sensitivity : 0.9618          
            Specificity : 0.4933          
         Pos Pred Value : 0.5456          
         Neg Pred Value : 0.9533          
             Prevalence : 0.3874          
         Detection Rate : 0.3726          
   Detection Prevalence : 0.6830          
      Balanced Accuracy : 0.7276          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 100

compute_aupr(image_level$filter_100, image_level$label, 'PR: Segmentations: zero all pixels < 100')

compute_auroc(image_level$filter_100, image_level$label, 'ROC: Segmentations: zero all pixels < 100')

confusion_matrix(image_level, 'filter_100')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 736 210
         1  91 313
                                         
               Accuracy : 0.777          
                 95% CI : (0.7539, 0.799)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : < 2.2e-16      
                                         
                  Kappa : 0.5098         
                                         
 Mcnemar's Test P-Value : 1.036e-11      
                                         
            Sensitivity : 0.5985         
            Specificity : 0.8900         
         Pos Pred Value : 0.7748         
         Neg Pred Value : 0.7780         
             Prevalence : 0.3874         
         Detection Rate : 0.2319         
   Detection Prevalence : 0.2993         
      Balanced Accuracy : 0.7442         
                                         
       'Positive' Class : 1              
                                         
compute_aupr(image_level$filter_100_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 100 and binarize')

compute_auroc(image_level$filter_100_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 100 and binarize')

confusion_matrix(image_level, 'filter_100_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 669 142
         1 158 381
                                          
               Accuracy : 0.7778          
                 95% CI : (0.7546, 0.7997)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : <2e-16          
                                          
                  Kappa : 0.5344          
                                          
 Mcnemar's Test P-Value : 0.3865          
                                          
            Sensitivity : 0.7285          
            Specificity : 0.8089          
         Pos Pred Value : 0.7069          
         Neg Pred Value : 0.8249          
             Prevalence : 0.3874          
         Detection Rate : 0.2822          
   Detection Prevalence : 0.3993          
      Balanced Accuracy : 0.7687          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_100_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 100 and skeletonize')

compute_auroc(image_level$filter_100_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 100 and skeletonize')

confusion_matrix(image_level, 'filter_100_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 632 144
         1 195 379
                                          
               Accuracy : 0.7489          
                 95% CI : (0.7249, 0.7718)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.4803          
                                          
 Mcnemar's Test P-Value : 0.006615        
                                          
            Sensitivity : 0.7247          
            Specificity : 0.7642          
         Pos Pred Value : 0.6603          
         Neg Pred Value : 0.8144          
             Prevalence : 0.3874          
         Detection Rate : 0.2807          
   Detection Prevalence : 0.4252          
      Balanced Accuracy : 0.7444          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 150

compute_aupr(image_level$filter_150, image_level$label, 'PR: Segmentations: zero all pixels < 150')

compute_auroc(image_level$filter_150, image_level$label, 'ROC: Segmentations: zero all pixels < 150')

confusion_matrix(image_level, 'filter_150')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 625 118
         1 202 405
                                          
               Accuracy : 0.763           
                 95% CI : (0.7393, 0.7854)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.5149          
                                          
 Mcnemar's Test P-Value : 3.487e-06       
                                          
            Sensitivity : 0.7744          
            Specificity : 0.7557          
         Pos Pred Value : 0.6672          
         Neg Pred Value : 0.8412          
             Prevalence : 0.3874          
         Detection Rate : 0.3000          
   Detection Prevalence : 0.4496          
      Balanced Accuracy : 0.7651          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_150_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 150 and binarize')

compute_auroc(image_level$filter_150_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 150 and binarize')

confusion_matrix(image_level, 'filter_150_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 702 167
         1 125 356
                                          
               Accuracy : 0.7837          
                 95% CI : (0.7608, 0.8054)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2e-16         
                                          
                  Kappa : 0.5375          
                                          
 Mcnemar's Test P-Value : 0.01642         
                                          
            Sensitivity : 0.6807          
            Specificity : 0.8489          
         Pos Pred Value : 0.7401          
         Neg Pred Value : 0.8078          
             Prevalence : 0.3874          
         Detection Rate : 0.2637          
   Detection Prevalence : 0.3563          
      Balanced Accuracy : 0.7648          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_150_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 150 and skeletonize')

compute_auroc(image_level$filter_150_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 150 and skeletonize')

confusion_matrix(image_level, 'filter_150_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 641 162
         1 186 361
                                         
               Accuracy : 0.7422         
                 95% CI : (0.718, 0.7654)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : <2e-16         
                                         
                  Kappa : 0.4614         
                                         
 Mcnemar's Test P-Value : 0.2176         
                                         
            Sensitivity : 0.6902         
            Specificity : 0.7751         
         Pos Pred Value : 0.6600         
         Neg Pred Value : 0.7983         
             Prevalence : 0.3874         
         Detection Rate : 0.2674         
   Detection Prevalence : 0.4052         
      Balanced Accuracy : 0.7327         
                                         
       'Positive' Class : 1              
                                         

Segmentations: zero all pixels < 200

compute_aupr(image_level$filter_200, image_level$label, 'PR: Segmentations: zero all pixels < 200')

compute_auroc(image_level$filter_200, image_level$label, 'ROC: Segmentations: zero all pixels < 200')

confusion_matrix(image_level, 'filter_200')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 548 121
         1 279 402
                                         
               Accuracy : 0.7037         
                 95% CI : (0.6786, 0.728)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : 1.64e-12       
                                         
                  Kappa : 0.4086         
                                         
 Mcnemar's Test P-Value : 4.16e-15       
                                         
            Sensitivity : 0.7686         
            Specificity : 0.6626         
         Pos Pred Value : 0.5903         
         Neg Pred Value : 0.8191         
             Prevalence : 0.3874         
         Detection Rate : 0.2978         
   Detection Prevalence : 0.5044         
      Balanced Accuracy : 0.7156         
                                         
       'Positive' Class : 1              
                                         
compute_aupr(image_level$filter_200_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 200 and binarize')

compute_auroc(image_level$filter_200_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 200 and binarize')

confusion_matrix(image_level, 'filter_200_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 661 233
         1 166 290
                                          
               Accuracy : 0.7044          
                 95% CI : (0.6793, 0.7287)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 1.083e-12       
                                          
                  Kappa : 0.3623          
                                          
 Mcnemar's Test P-Value : 0.0009527       
                                          
            Sensitivity : 0.5545          
            Specificity : 0.7993          
         Pos Pred Value : 0.6360          
         Neg Pred Value : 0.7394          
             Prevalence : 0.3874          
         Detection Rate : 0.2148          
   Detection Prevalence : 0.3378          
      Balanced Accuracy : 0.6769          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_200_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 200 and skeletonize')

compute_auroc(image_level$filter_200_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 200 and skeletonize')

confusion_matrix(image_level, 'filter_200_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 766 365
         1  61 158
                                          
               Accuracy : 0.6844          
                 95% CI : (0.6589, 0.7092)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 2.354e-08       
                                          
                  Kappa : 0.2557          
                                          
 Mcnemar's Test P-Value : < 2.2e-16       
                                          
            Sensitivity : 0.3021          
            Specificity : 0.9262          
         Pos Pred Value : 0.7215          
         Neg Pred Value : 0.6773          
             Prevalence : 0.3874          
         Detection Rate : 0.1170          
   Detection Prevalence : 0.1622          
      Balanced Accuracy : 0.6142          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 210

compute_aupr(image_level$filter_210, image_level$label, 'PR: Segmentations: zero all pixels < 210')

compute_auroc(image_level$filter_210, image_level$label, 'ROC: Segmentations: zero all pixels < 210')

confusion_matrix(image_level, 'filter_210')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 712 326
         1 115 197
                                          
               Accuracy : 0.6733          
                 95% CI : (0.6476, 0.6983)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 2.096e-06       
                                          
                  Kappa : 0.2566          
                                          
 Mcnemar's Test P-Value : < 2.2e-16       
                                          
            Sensitivity : 0.3767          
            Specificity : 0.8609          
         Pos Pred Value : 0.6314          
         Neg Pred Value : 0.6859          
             Prevalence : 0.3874          
         Detection Rate : 0.1459          
   Detection Prevalence : 0.2311          
      Balanced Accuracy : 0.6188          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_210_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 210 and binarize')

compute_auroc(image_level$filter_210_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 210 and binarize')

confusion_matrix(image_level, 'filter_210_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 639 203
         1 188 320
                                          
               Accuracy : 0.7104          
                 95% CI : (0.6854, 0.7345)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 3.463e-14       
                                          
                  Kappa : 0.3866          
                                          
 Mcnemar's Test P-Value : 0.4789          
                                          
            Sensitivity : 0.6119          
            Specificity : 0.7727          
         Pos Pred Value : 0.6299          
         Neg Pred Value : 0.7589          
             Prevalence : 0.3874          
         Detection Rate : 0.2370          
   Detection Prevalence : 0.3763          
      Balanced Accuracy : 0.6923          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_210_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 210 and skeletonize')

compute_auroc(image_level$filter_210_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 210 and skeletonize')

confusion_matrix(image_level, 'filter_210_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 562 160
         1 265 363
                                          
               Accuracy : 0.6852          
                 95% CI : (0.6597, 0.7099)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 1.699e-08       
                                          
                  Kappa : 0.3603          
                                          
 Mcnemar's Test P-Value : 4.541e-07       
                                          
            Sensitivity : 0.6941          
            Specificity : 0.6796          
         Pos Pred Value : 0.5780          
         Neg Pred Value : 0.7784          
             Prevalence : 0.3874          
         Detection Rate : 0.2689          
   Detection Prevalence : 0.4652          
      Balanced Accuracy : 0.6868          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 220

compute_aupr(image_level$filter_220, image_level$label, 'PR: Segmentations: zero all pixels < 220')

compute_auroc(image_level$filter_220, image_level$label, 'ROC: Segmentations: zero all pixels < 220')

confusion_matrix(image_level, 'filter_220')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 576 224
         1 251 299
                                         
               Accuracy : 0.6481         
                 95% CI : (0.622, 0.6736)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : 0.003819       
                                         
                  Kappa : 0.2657         
                                         
 Mcnemar's Test P-Value : 0.232884       
                                         
            Sensitivity : 0.5717         
            Specificity : 0.6965         
         Pos Pred Value : 0.5436         
         Neg Pred Value : 0.7200         
             Prevalence : 0.3874         
         Detection Rate : 0.2215         
   Detection Prevalence : 0.4074         
      Balanced Accuracy : 0.6341         
                                         
       'Positive' Class : 1              
                                         
compute_aupr(image_level$filter_220_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 220 and binarize')

compute_auroc(image_level$filter_220_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 220 and binarize')

confusion_matrix(image_level, 'filter_220_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 545 162
         1 282 361
                                          
               Accuracy : 0.6711          
                 95% CI : (0.6453, 0.6961)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 4.710e-06       
                                          
                  Kappa : 0.3351          
                                          
 Mcnemar's Test P-Value : 1.628e-08       
                                          
            Sensitivity : 0.6902          
            Specificity : 0.6590          
         Pos Pred Value : 0.5614          
         Neg Pred Value : 0.7709          
             Prevalence : 0.3874          
         Detection Rate : 0.2674          
   Detection Prevalence : 0.4763          
      Balanced Accuracy : 0.6746          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_220_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 220 and skeletonize')

compute_auroc(image_level$filter_220_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 220 and skeletonize')

confusion_matrix(image_level, 'filter_220_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 800 450
         1  27  73
                                          
               Accuracy : 0.6467          
                 95% CI : (0.6205, 0.6722)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 0.005317        
                                          
                  Kappa : 0.1256          
                                          
 Mcnemar's Test P-Value : < 2.2e-16       
                                          
            Sensitivity : 0.13958         
            Specificity : 0.96735         
         Pos Pred Value : 0.73000         
         Neg Pred Value : 0.64000         
             Prevalence : 0.38741         
         Detection Rate : 0.05407         
   Detection Prevalence : 0.07407         
      Balanced Accuracy : 0.55347         
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 230

compute_aupr(image_level$filter_230, image_level$label, 'PR: Segmentations: zero all pixels < 230')

compute_auroc(image_level$filter_230, image_level$label, 'ROC: Segmentations: zero all pixels < 230')

confusion_matrix(image_level, 'filter_230')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 464 160
         1 363 363
                                         
               Accuracy : 0.6126         
                 95% CI : (0.586, 0.6387)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : 0.512          
                                         
                  Kappa : 0.2381         
                                         
 Mcnemar's Test P-Value : <2e-16         
                                         
            Sensitivity : 0.6941         
            Specificity : 0.5611         
         Pos Pred Value : 0.5000         
         Neg Pred Value : 0.7436         
             Prevalence : 0.3874         
         Detection Rate : 0.2689         
   Detection Prevalence : 0.5378         
      Balanced Accuracy : 0.6276         
                                         
       'Positive' Class : 1              
                                         
compute_aupr(image_level$filter_230_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 230 and binarize')

compute_auroc(image_level$filter_230_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 230 and binarize')

confusion_matrix(image_level, 'filter_230_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 246  46
         1 581 477
                                          
               Accuracy : 0.5356          
                 95% CI : (0.5085, 0.5624)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 1               
                                          
                  Kappa : 0.1764          
                                          
 Mcnemar's Test P-Value : <2e-16          
                                          
            Sensitivity : 0.9120          
            Specificity : 0.2975          
         Pos Pred Value : 0.4509          
         Neg Pred Value : 0.8425          
             Prevalence : 0.3874          
         Detection Rate : 0.3533          
   Detection Prevalence : 0.7837          
      Balanced Accuracy : 0.6048          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_230_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 230 and skeletonize')

compute_auroc(image_level$filter_230_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 230 and skeletonize')

confusion_matrix(image_level, 'filter_230_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 365 110
         1 462 413
                                          
               Accuracy : 0.5763          
                 95% CI : (0.5494, 0.6028)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 0.997           
                                          
                  Kappa : 0.2056          
                                          
 Mcnemar's Test P-Value : <2e-16          
                                          
            Sensitivity : 0.7897          
            Specificity : 0.4414          
         Pos Pred Value : 0.4720          
         Neg Pred Value : 0.7684          
             Prevalence : 0.3874          
         Detection Rate : 0.3059          
   Detection Prevalence : 0.6481          
      Balanced Accuracy : 0.6155          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 240

compute_aupr(image_level$filter_240, image_level$label, 'PR: Segmentations: zero all pixels < 240')

compute_auroc(image_level$filter_240, image_level$label, 'ROC: Segmentations: zero all pixels < 240')

confusion_matrix(image_level, 'filter_240')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 414 183
         1 413 340
                                          
               Accuracy : 0.5585          
                 95% CI : (0.5316, 0.5852)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 1               
                                          
                  Kappa : 0.1394          
                                          
 Mcnemar's Test P-Value : <2e-16          
                                          
            Sensitivity : 0.6501          
            Specificity : 0.5006          
         Pos Pred Value : 0.4515          
         Neg Pred Value : 0.6935          
             Prevalence : 0.3874          
         Detection Rate : 0.2519          
   Detection Prevalence : 0.5578          
      Balanced Accuracy : 0.5754          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_240_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 240 and binarize')

compute_auroc(image_level$filter_240_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 240 and binarize')

confusion_matrix(image_level, 'filter_240_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 258  83
         1 569 440
                                       
               Accuracy : 0.517        
                 95% CI : (0.49, 0.544)
    No Information Rate : 0.6126       
    P-Value [Acc > NIR] : 1            
                                       
                  Kappa : 0.1309       
                                       
 Mcnemar's Test P-Value : <2e-16       
                                       
            Sensitivity : 0.8413       
            Specificity : 0.3120       
         Pos Pred Value : 0.4361       
         Neg Pred Value : 0.7566       
             Prevalence : 0.3874       
         Detection Rate : 0.3259       
   Detection Prevalence : 0.7474       
      Balanced Accuracy : 0.5766       
                                       
       'Positive' Class : 1            
                                       
compute_aupr(image_level$filter_240_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 240 and skeletonize')

compute_auroc(image_level$filter_240_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 240 and skeletonize')

confusion_matrix(image_level, 'filter_240_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 279 103
         1 548 420
                                          
               Accuracy : 0.5178          
                 95% CI : (0.4907, 0.5447)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 1               
                                          
                  Kappa : 0.1214          
                                          
 Mcnemar's Test P-Value : <2e-16          
                                          
            Sensitivity : 0.8031          
            Specificity : 0.3374          
         Pos Pred Value : 0.4339          
         Neg Pred Value : 0.7304          
             Prevalence : 0.3874          
         Detection Rate : 0.3111          
   Detection Prevalence : 0.7170          
      Balanced Accuracy : 0.5702          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 250

compute_aupr(image_level$filter_250, image_level$label, 'PR: Segmentations: zero all pixels < 250')

compute_auroc(image_level$filter_250, image_level$label, 'ROC: Segmentations: zero all pixels < 250')

confusion_matrix(image_level, 'filter_250')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0  51   9
         1 776 514
                                         
               Accuracy : 0.4185         
                 95% CI : (0.392, 0.4454)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : 1              
                                         
                  Kappa : 0.035          
                                         
 Mcnemar's Test P-Value : <2e-16         
                                         
            Sensitivity : 0.98279        
            Specificity : 0.06167        
         Pos Pred Value : 0.39845        
         Neg Pred Value : 0.85000        
             Prevalence : 0.38741        
         Detection Rate : 0.38074        
   Detection Prevalence : 0.95556        
      Balanced Accuracy : 0.52223        
                                         
       'Positive' Class : 1              
                                         
compute_aupr(image_level$filter_250_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 250 and binarize')

compute_auroc(image_level$filter_250_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 250 and binarize')

confusion_matrix(image_level, 'filter_250_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0  50   8
         1 777 515
                                         
               Accuracy : 0.4185         
                 95% CI : (0.392, 0.4454)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : 1              
                                         
                  Kappa : 0.0356         
                                         
 Mcnemar's Test P-Value : <2e-16         
                                         
            Sensitivity : 0.98470        
            Specificity : 0.06046        
         Pos Pred Value : 0.39861        
         Neg Pred Value : 0.86207        
             Prevalence : 0.38741        
         Detection Rate : 0.38148        
   Detection Prevalence : 0.95704        
      Balanced Accuracy : 0.52258        
                                         
       'Positive' Class : 1              
                                         
compute_aupr(image_level$filter_250_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 250 and skeletonize')

compute_auroc(image_level$filter_250_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 250 and skeletonize')

confusion_matrix(image_level, 'filter_250_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0  53   9
         1 774 514
                                          
               Accuracy : 0.42            
                 95% CI : (0.3935, 0.4468)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 1               
                                          
                  Kappa : 0.0369          
                                          
 Mcnemar's Test P-Value : <2e-16          
                                          
            Sensitivity : 0.98279         
            Specificity : 0.06409         
         Pos Pred Value : 0.39907         
         Neg Pred Value : 0.85484         
             Prevalence : 0.38741         
         Detection Rate : 0.38074         
   Detection Prevalence : 0.95407         
      Balanced Accuracy : 0.52344         
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 257

compute_aupr(image_level$filter_257, image_level$label, 'PR: Segmentations: zero all pixels < 257')

compute_auroc(image_level$filter_257, image_level$label, 'ROC: Segmentations: zero all pixels < 257')

confusion_matrix(image_level, 'filter_257')
Warning in confusionMatrix.default(preds, df$label, positive = "1") :
  Levels are not in the same order for reference and data. Refactoring data to match.
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 827 523
         1   0   0
                                         
               Accuracy : 0.6126         
                 95% CI : (0.586, 0.6387)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : 0.512          
                                         
                  Kappa : 0              
                                         
 Mcnemar's Test P-Value : <2e-16         
                                         
            Sensitivity : 0.0000         
            Specificity : 1.0000         
         Pos Pred Value :    NaN         
         Neg Pred Value : 0.6126         
             Prevalence : 0.3874         
         Detection Rate : 0.0000         
   Detection Prevalence : 0.0000         
      Balanced Accuracy : 0.5000         
                                         
       'Positive' Class : 1              
                                         
compute_aupr(image_level$filter_257_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 257 and binarize')

compute_auroc(image_level$filter_257_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 257 and binarize')

confusion_matrix(image_level, 'filter_257_binarized')
Warning in confusionMatrix.default(preds, df$label, positive = "1") :
  Levels are not in the same order for reference and data. Refactoring data to match.
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 827 523
         1   0   0
                                         
               Accuracy : 0.6126         
                 95% CI : (0.586, 0.6387)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : 0.512          
                                         
                  Kappa : 0              
                                         
 Mcnemar's Test P-Value : <2e-16         
                                         
            Sensitivity : 0.0000         
            Specificity : 1.0000         
         Pos Pred Value :    NaN         
         Neg Pred Value : 0.6126         
             Prevalence : 0.3874         
         Detection Rate : 0.0000         
   Detection Prevalence : 0.0000         
      Balanced Accuracy : 0.5000         
                                         
       'Positive' Class : 1              
                                         
compute_aupr(image_level$filter_257_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 257 and skeletonize')

compute_auroc(image_level$filter_257_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 257 and skeletonize')

confusion_matrix(image_level, 'filter_257_skeletonized')
Warning in confusionMatrix.default(preds, df$label, positive = "1") :
  Levels are not in the same order for reference and data. Refactoring data to match.
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 827 523
         1   0   0
                                         
               Accuracy : 0.6126         
                 95% CI : (0.586, 0.6387)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : 0.512          
                                         
                  Kappa : 0              
                                         
 Mcnemar's Test P-Value : <2e-16         
                                         
            Sensitivity : 0.0000         
            Specificity : 1.0000         
         Pos Pred Value :    NaN         
         Neg Pred Value : 0.6126         
             Prevalence : 0.3874         
         Detection Rate : 0.0000         
   Detection Prevalence : 0.0000         
      Balanced Accuracy : 0.5000         
                                         
       'Positive' Class : 1              
                                         

Segmentations: zero all pixels > 10

compute_aupr(image_level$filter_10, image_level$label, 'PR: Segmentations: zero all pixels < 10')

compute_auroc(image_level$filter_10, image_level$label, 'ROC: Segmentations: zero all pixels < 10')

confusion_matrix(image_level, 'filter_10')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 763 196
         1  64 327
                                          
               Accuracy : 0.8074          
                 95% CI : (0.7853, 0.8281)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2e-16         
                                          
                  Kappa : 0.5745          
                                          
 Mcnemar's Test P-Value : 4.5e-16         
                                          
            Sensitivity : 0.6252          
            Specificity : 0.9226          
         Pos Pred Value : 0.8363          
         Neg Pred Value : 0.7956          
             Prevalence : 0.3874          
         Detection Rate : 0.2422          
   Detection Prevalence : 0.2896          
      Balanced Accuracy : 0.7739          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_10_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 10 and binarize')

compute_auroc(image_level$filter_10_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 10 and binarize')

confusion_matrix(image_level, 'filter_10_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 621   7
         1 206 516
                                          
               Accuracy : 0.8422          
                 95% CI : (0.8217, 0.8613)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.6893          
                                          
 Mcnemar's Test P-Value : < 2.2e-16       
                                          
            Sensitivity : 0.9866          
            Specificity : 0.7509          
         Pos Pred Value : 0.7147          
         Neg Pred Value : 0.9889          
             Prevalence : 0.3874          
         Detection Rate : 0.3822          
   Detection Prevalence : 0.5348          
      Balanced Accuracy : 0.8688          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_10_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 10 and skeletonize')

compute_auroc(image_level$filter_10_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 10 and skeletonize')

confusion_matrix(image_level, 'filter_10_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 696  43
         1 131 480
                                          
               Accuracy : 0.8711          
                 95% CI : (0.8521, 0.8885)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.7366          
                                          
 Mcnemar's Test P-Value : 4.24e-11        
                                          
            Sensitivity : 0.9178          
            Specificity : 0.8416          
         Pos Pred Value : 0.7856          
         Neg Pred Value : 0.9418          
             Prevalence : 0.3874          
         Detection Rate : 0.3556          
   Detection Prevalence : 0.4526          
      Balanced Accuracy : 0.8797          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 75 and > 150

compute_aupr(image_level$filter_75, image_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150')

compute_auroc(image_level$filter_75, image_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150')

confusion_matrix(image_level, 'filter_75')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 597  77
         1 230 446
                                          
               Accuracy : 0.7726          
                 95% CI : (0.7493, 0.7947)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.5453          
                                          
 Mcnemar's Test P-Value : < 2.2e-16       
                                          
            Sensitivity : 0.8528          
            Specificity : 0.7219          
         Pos Pred Value : 0.6598          
         Neg Pred Value : 0.8858          
             Prevalence : 0.3874          
         Detection Rate : 0.3304          
   Detection Prevalence : 0.5007          
      Balanced Accuracy : 0.7873          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_75_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150 and binarize')

compute_auroc(image_level$filter_75_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150 and binarize')

confusion_matrix(image_level, 'filter_75_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 545  69
         1 282 454
                                          
               Accuracy : 0.74            
                 95% CI : (0.7157, 0.7632)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.4904          
                                          
 Mcnemar's Test P-Value : < 2.2e-16       
                                          
            Sensitivity : 0.8681          
            Specificity : 0.6590          
         Pos Pred Value : 0.6168          
         Neg Pred Value : 0.8876          
             Prevalence : 0.3874          
         Detection Rate : 0.3363          
   Detection Prevalence : 0.5452          
      Balanced Accuracy : 0.7635          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_75_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150 and skeletonize')

compute_auroc(image_level$filter_75_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150 and skeletonize')

confusion_matrix(image_level, 'filter_75_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 726 224
         1 101 299
                                          
               Accuracy : 0.7593          
                 95% CI : (0.7355, 0.7819)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.4699          
                                          
 Mcnemar's Test P-Value : 1.312e-11       
                                          
            Sensitivity : 0.5717          
            Specificity : 0.8779          
         Pos Pred Value : 0.7475          
         Neg Pred Value : 0.7642          
             Prevalence : 0.3874          
         Detection Rate : 0.2215          
   Detection Prevalence : 0.2963          
      Balanced Accuracy : 0.7248          
                                          
       'Positive' Class : 1               
                                          

Subject-level Analysis

RetCam Images

compute_aupr(subject_level$retcam, subject_level$label, 'PR: Raw RetCam Images')

compute_auroc(subject_level$retcam, subject_level$label, 'ROC: Raw RetCam Images')

confusion_matrix(subject_level, 'retcam')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43  0
         1  0 27
                                     
               Accuracy : 1          
                 95% CI : (0.9487, 1)
    No Information Rate : 0.6143     
    P-Value [Acc > NIR] : 1.534e-15  
                                     
                  Kappa : 1          
                                     
 Mcnemar's Test P-Value : NA         
                                     
            Sensitivity : 1.0000     
            Specificity : 1.0000     
         Pos Pred Value : 1.0000     
         Neg Pred Value : 1.0000     
             Prevalence : 0.3857     
         Detection Rate : 0.3857     
   Detection Prevalence : 0.3857     
      Balanced Accuracy : 1.0000     
                                     
       'Positive' Class : 1          
                                     
compute_aupr(subject_level$retcam_random, subject_level$label, 'PR: Raw RetCam Images - Shuffled Labels')

compute_auroc(subject_level$retcam_random, subject_level$label, 'ROC: Raw RetCam Images - Shuffled Labels')

confusion_matrix(subject_level, 'retcam_random')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 24  2
         1 19 25
                                          
               Accuracy : 0.7             
                 95% CI : (0.5787, 0.8038)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.0869051       
                                          
                  Kappa : 0.4333          
                                          
 Mcnemar's Test P-Value : 0.0004803       
                                          
            Sensitivity : 0.9259          
            Specificity : 0.5581          
         Pos Pred Value : 0.5682          
         Neg Pred Value : 0.9231          
             Prevalence : 0.3857          
         Detection Rate : 0.3571          
   Detection Prevalence : 0.6286          
      Balanced Accuracy : 0.7420          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 0

compute_aupr(subject_level$filter_0, subject_level$label, 'PR: Segmentations: zero all pixels < 0')

compute_auroc(subject_level$filter_0, subject_level$label, 'ROC: Segmentations: zero all pixels < 0')

confusion_matrix(subject_level, 'filter_0')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 42  2
         1  1 25
                                          
               Accuracy : 0.9571          
                 95% CI : (0.8798, 0.9911)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 2.232e-11       
                                          
                  Kappa : 0.9089          
                                          
 Mcnemar's Test P-Value : 1               
                                          
            Sensitivity : 0.9259          
            Specificity : 0.9767          
         Pos Pred Value : 0.9615          
         Neg Pred Value : 0.9545          
             Prevalence : 0.3857          
         Detection Rate : 0.3571          
   Detection Prevalence : 0.3714          
      Balanced Accuracy : 0.9513          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_0_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 0 and binarize')

compute_auroc(subject_level$filter_0_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 0 and binarize')

confusion_matrix(subject_level, 'filter_0_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 41  0
         1  2 27
                                          
               Accuracy : 0.9714          
                 95% CI : (0.9006, 0.9965)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 1.53e-12        
                                          
                  Kappa : 0.9405          
                                          
 Mcnemar's Test P-Value : 0.4795          
                                          
            Sensitivity : 1.0000          
            Specificity : 0.9535          
         Pos Pred Value : 0.9310          
         Neg Pred Value : 1.0000          
             Prevalence : 0.3857          
         Detection Rate : 0.3857          
   Detection Prevalence : 0.4143          
      Balanced Accuracy : 0.9767          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_0_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 0 and skeletonize')

compute_auroc(subject_level$filter_0_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 0 and skeletonize')

confusion_matrix(subject_level, 'filter_0_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 40  0
         1  3 27
                                          
               Accuracy : 0.9571          
                 95% CI : (0.8798, 0.9911)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 2.232e-11       
                                          
                  Kappa : 0.9114          
                                          
 Mcnemar's Test P-Value : 0.2482          
                                          
            Sensitivity : 1.0000          
            Specificity : 0.9302          
         Pos Pred Value : 0.9000          
         Neg Pred Value : 1.0000          
             Prevalence : 0.3857          
         Detection Rate : 0.3857          
   Detection Prevalence : 0.4286          
      Balanced Accuracy : 0.9651          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_0_random, subject_level$label, 'PR: Segmentations: zero all pixels < 0 - Shuffled Labels')

compute_auroc(subject_level$filter_0_random, subject_level$label, 'ROC: Segmentations: zero all pixels < 0 - Shuffled Labels')

confusion_matrix(subject_level, 'filter_0_random')
Warning in confusionMatrix.default(preds, df$label, positive = "1") :
  Levels are not in the same order for reference and data. Refactoring data to match.
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43 27
         1  0  0
                                          
               Accuracy : 0.6143          
                 95% CI : (0.4903, 0.7283)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.5525          
                                          
                  Kappa : 0               
                                          
 Mcnemar's Test P-Value : 5.624e-07       
                                          
            Sensitivity : 0.0000          
            Specificity : 1.0000          
         Pos Pred Value :    NaN          
         Neg Pred Value : 0.6143          
             Prevalence : 0.3857          
         Detection Rate : 0.0000          
   Detection Prevalence : 0.0000          
      Balanced Accuracy : 0.5000          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 50

compute_aupr(subject_level$filter_50, subject_level$label, 'PR: Segmentations: zero all pixels < 50')

compute_auroc(subject_level$filter_50, subject_level$label, 'ROC: Segmentations: zero all pixels < 50')

confusion_matrix(subject_level, 'filter_50')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 36  0
         1  7 27
                                          
               Accuracy : 0.9             
                 95% CI : (0.8048, 0.9588)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 8.517e-08       
                                          
                  Kappa : 0.7987          
                                          
 Mcnemar's Test P-Value : 0.02334         
                                          
            Sensitivity : 1.0000          
            Specificity : 0.8372          
         Pos Pred Value : 0.7941          
         Neg Pred Value : 1.0000          
             Prevalence : 0.3857          
         Detection Rate : 0.3857          
   Detection Prevalence : 0.4857          
      Balanced Accuracy : 0.9186          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_50_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 50 and binarize')

compute_auroc(subject_level$filter_50_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 50 and binarize')

confusion_matrix(subject_level, 'filter_0_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 41  0
         1  2 27
                                          
               Accuracy : 0.9714          
                 95% CI : (0.9006, 0.9965)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 1.53e-12        
                                          
                  Kappa : 0.9405          
                                          
 Mcnemar's Test P-Value : 0.4795          
                                          
            Sensitivity : 1.0000          
            Specificity : 0.9535          
         Pos Pred Value : 0.9310          
         Neg Pred Value : 1.0000          
             Prevalence : 0.3857          
         Detection Rate : 0.3857          
   Detection Prevalence : 0.4143          
      Balanced Accuracy : 0.9767          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_50_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 50 and skeletonize')

compute_auroc(subject_level$filter_50_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 50 and skeletonize')

confusion_matrix(subject_level, 'filter_50_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 19  0
         1 24 27
                                         
               Accuracy : 0.6571         
                 95% CI : (0.534, 0.7665)
    No Information Rate : 0.6143         
    P-Value [Acc > NIR] : 0.2718         
                                         
                  Kappa : 0.3792         
                                         
 Mcnemar's Test P-Value : 2.668e-06      
                                         
            Sensitivity : 1.0000         
            Specificity : 0.4419         
         Pos Pred Value : 0.5294         
         Neg Pred Value : 1.0000         
             Prevalence : 0.3857         
         Detection Rate : 0.3857         
   Detection Prevalence : 0.7286         
      Balanced Accuracy : 0.7209         
                                         
       'Positive' Class : 1              
                                         

Segmentations: zero all pixels < 100

compute_aupr(subject_level$filter_100, subject_level$label, 'PR: Segmentations: zero all pixels < 100')

compute_auroc(subject_level$filter_100, subject_level$label, 'ROC: Segmentations: zero all pixels < 100')

confusion_matrix(subject_level, 'filter_100')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43 10
         1  0 17
                                          
               Accuracy : 0.8571          
                 95% CI : (0.7529, 0.9293)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 7.748e-06       
                                          
                  Kappa : 0.6762          
                                          
 Mcnemar's Test P-Value : 0.004427        
                                          
            Sensitivity : 0.6296          
            Specificity : 1.0000          
         Pos Pred Value : 1.0000          
         Neg Pred Value : 0.8113          
             Prevalence : 0.3857          
         Detection Rate : 0.2429          
   Detection Prevalence : 0.2429          
      Balanced Accuracy : 0.8148          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_100_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 100 and binarize')

compute_auroc(subject_level$filter_100_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 100 and binarize')

confusion_matrix(subject_level, 'filter_100_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 39  4
         1  4 23
                                          
               Accuracy : 0.8857          
                 95% CI : (0.7872, 0.9493)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 4.352e-07       
                                          
                  Kappa : 0.7588          
                                          
 Mcnemar's Test P-Value : 1               
                                          
            Sensitivity : 0.8519          
            Specificity : 0.9070          
         Pos Pred Value : 0.8519          
         Neg Pred Value : 0.9070          
             Prevalence : 0.3857          
         Detection Rate : 0.3286          
   Detection Prevalence : 0.3857          
      Balanced Accuracy : 0.8794          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_100_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 100 and skeletonize')

compute_auroc(subject_level$filter_100_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 100 and skeletonize')

confusion_matrix(subject_level, 'filter_100_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 38  3
         1  5 24
                                          
               Accuracy : 0.8857          
                 95% CI : (0.7872, 0.9493)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 4.352e-07       
                                          
                  Kappa : 0.7621          
                                          
 Mcnemar's Test P-Value : 0.7237          
                                          
            Sensitivity : 0.8889          
            Specificity : 0.8837          
         Pos Pred Value : 0.8276          
         Neg Pred Value : 0.9268          
             Prevalence : 0.3857          
         Detection Rate : 0.3429          
   Detection Prevalence : 0.4143          
      Balanced Accuracy : 0.8863          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 150

compute_aupr(subject_level$filter_150, subject_level$label, 'PR: Segmentations: zero all pixels < 150')

compute_auroc(subject_level$filter_150, subject_level$label, 'ROC: Segmentations: zero all pixels < 150')

confusion_matrix(subject_level, 'filter_150')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 37  0
         1  6 27
                                          
               Accuracy : 0.9143          
                 95% CI : (0.8227, 0.9679)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 1.438e-08       
                                          
                  Kappa : 0.8263          
                                          
 Mcnemar's Test P-Value : 0.04123         
                                          
            Sensitivity : 1.0000          
            Specificity : 0.8605          
         Pos Pred Value : 0.8182          
         Neg Pred Value : 1.0000          
             Prevalence : 0.3857          
         Detection Rate : 0.3857          
   Detection Prevalence : 0.4714          
      Balanced Accuracy : 0.9302          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_150_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 150 and binarize')

compute_auroc(subject_level$filter_150_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 150 and binarize')

confusion_matrix(subject_level, 'filter_150_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43  5
         1  0 22
                                          
               Accuracy : 0.9286          
                 95% CI : (0.8411, 0.9764)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 2.054e-09       
                                          
                  Kappa : 0.8439          
                                          
 Mcnemar's Test P-Value : 0.07364         
                                          
            Sensitivity : 0.8148          
            Specificity : 1.0000          
         Pos Pred Value : 1.0000          
         Neg Pred Value : 0.8958          
             Prevalence : 0.3857          
         Detection Rate : 0.3143          
   Detection Prevalence : 0.3143          
      Balanced Accuracy : 0.9074          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_150_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 150 and skeletonize')

compute_auroc(subject_level$filter_150_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 150 and skeletonize')

confusion_matrix(subject_level, 'filter_150_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 38  3
         1  5 24
                                          
               Accuracy : 0.8857          
                 95% CI : (0.7872, 0.9493)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 4.352e-07       
                                          
                  Kappa : 0.7621          
                                          
 Mcnemar's Test P-Value : 0.7237          
                                          
            Sensitivity : 0.8889          
            Specificity : 0.8837          
         Pos Pred Value : 0.8276          
         Neg Pred Value : 0.9268          
             Prevalence : 0.3857          
         Detection Rate : 0.3429          
   Detection Prevalence : 0.4143          
      Balanced Accuracy : 0.8863          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 200

compute_aupr(subject_level$filter_200, subject_level$label, 'PR: Segmentations: zero all pixels < 200')

compute_auroc(subject_level$filter_200, subject_level$label, 'ROC: Segmentations: zero all pixels < 200')

confusion_matrix(subject_level, 'filter_200')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 34  4
         1  9 23
                                          
               Accuracy : 0.8143          
                 95% CI : (0.7034, 0.8972)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.0002607       
                                          
                  Kappa : 0.6211          
                                          
 Mcnemar's Test P-Value : 0.2672575       
                                          
            Sensitivity : 0.8519          
            Specificity : 0.7907          
         Pos Pred Value : 0.7187          
         Neg Pred Value : 0.8947          
             Prevalence : 0.3857          
         Detection Rate : 0.3286          
   Detection Prevalence : 0.4571          
      Balanced Accuracy : 0.8213          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_200_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 200 and binarize')

compute_auroc(subject_level$filter_200_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 200 and binarize')

confusion_matrix(subject_level, 'filter_200_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 40 12
         1  3 15
                                          
               Accuracy : 0.7857          
                 95% CI : (0.6713, 0.8748)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.001729        
                                          
                  Kappa : 0.5179          
                                          
 Mcnemar's Test P-Value : 0.038867        
                                          
            Sensitivity : 0.5556          
            Specificity : 0.9302          
         Pos Pred Value : 0.8333          
         Neg Pred Value : 0.7692          
             Prevalence : 0.3857          
         Detection Rate : 0.2143          
   Detection Prevalence : 0.2571          
      Balanced Accuracy : 0.7429          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_200_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 200 and skeletonize')

compute_auroc(subject_level$filter_200_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 200 and skeletonize')

confusion_matrix(subject_level, 'filter_200_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43 25
         1  0  2
                                          
               Accuracy : 0.6429          
                 95% CI : (0.5193, 0.7539)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.3595          
                                          
                  Kappa : 0.0895          
                                          
 Mcnemar's Test P-Value : 1.587e-06       
                                          
            Sensitivity : 0.07407         
            Specificity : 1.00000         
         Pos Pred Value : 1.00000         
         Neg Pred Value : 0.63235         
             Prevalence : 0.38571         
         Detection Rate : 0.02857         
   Detection Prevalence : 0.02857         
      Balanced Accuracy : 0.53704         
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 210

compute_aupr(subject_level$filter_210, subject_level$label, 'PR: Segmentations: zero all pixels < 210')

compute_auroc(subject_level$filter_210, subject_level$label, 'ROC: Segmentations: zero all pixels < 210')

confusion_matrix(subject_level, 'filter_210')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43 20
         1  0  7
                                         
               Accuracy : 0.7143         
                 95% CI : (0.5938, 0.816)
    No Information Rate : 0.6143         
    P-Value [Acc > NIR] : 0.0532         
                                         
                  Kappa : 0.3007         
                                         
 Mcnemar's Test P-Value : 2.152e-05      
                                         
            Sensitivity : 0.2593         
            Specificity : 1.0000         
         Pos Pred Value : 1.0000         
         Neg Pred Value : 0.6825         
             Prevalence : 0.3857         
         Detection Rate : 0.1000         
   Detection Prevalence : 0.1000         
      Balanced Accuracy : 0.6296         
                                         
       'Positive' Class : 1              
                                         
compute_aupr(subject_level$filter_210_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 210 and binarize')

compute_auroc(subject_level$filter_210_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 210 and binarize')

confusion_matrix(subject_level, 'filter_210_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 39  9
         1  4 18
                                          
               Accuracy : 0.8143          
                 95% CI : (0.7034, 0.8972)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.0002607       
                                          
                  Kappa : 0.5941          
                                          
 Mcnemar's Test P-Value : 0.2672575       
                                          
            Sensitivity : 0.6667          
            Specificity : 0.9070          
         Pos Pred Value : 0.8182          
         Neg Pred Value : 0.8125          
             Prevalence : 0.3857          
         Detection Rate : 0.2571          
   Detection Prevalence : 0.3143          
      Balanced Accuracy : 0.7868          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_210_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 210 and skeletonize')

compute_auroc(subject_level$filter_210_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 210 and skeletonize')

confusion_matrix(subject_level, 'filter_210_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 35  7
         1  8 20
                                          
               Accuracy : 0.7857          
                 95% CI : (0.6713, 0.8748)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.001729        
                                          
                  Kappa : 0.5509          
                                          
 Mcnemar's Test P-Value : 1.000000        
                                          
            Sensitivity : 0.7407          
            Specificity : 0.8140          
         Pos Pred Value : 0.7143          
         Neg Pred Value : 0.8333          
             Prevalence : 0.3857          
         Detection Rate : 0.2857          
   Detection Prevalence : 0.4000          
      Balanced Accuracy : 0.7773          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 220

compute_aupr(subject_level$filter_220, subject_level$label, 'PR: Segmentations: zero all pixels < 220')

compute_auroc(subject_level$filter_220, subject_level$label, 'ROC: Segmentations: zero all pixels < 220')

confusion_matrix(subject_level, 'filter_220')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 34 12
         1  9 15
                                          
               Accuracy : 0.7             
                 95% CI : (0.5787, 0.8038)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.08691         
                                          
                  Kappa : 0.3536          
                                          
 Mcnemar's Test P-Value : 0.66252         
                                          
            Sensitivity : 0.5556          
            Specificity : 0.7907          
         Pos Pred Value : 0.6250          
         Neg Pred Value : 0.7391          
             Prevalence : 0.3857          
         Detection Rate : 0.2143          
   Detection Prevalence : 0.3429          
      Balanced Accuracy : 0.6731          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_220_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 220 and binarize')

compute_auroc(subject_level$filter_220_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 220 and binarize')

confusion_matrix(subject_level, 'filter_220_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 31  7
         1 12 20
                                        
               Accuracy : 0.7286        
                 95% CI : (0.609, 0.828)
    No Information Rate : 0.6143        
    P-Value [Acc > NIR] : 0.03066       
                                        
                  Kappa : 0.4463        
                                        
 Mcnemar's Test P-Value : 0.35880       
                                        
            Sensitivity : 0.7407        
            Specificity : 0.7209        
         Pos Pred Value : 0.6250        
         Neg Pred Value : 0.8158        
             Prevalence : 0.3857        
         Detection Rate : 0.2857        
   Detection Prevalence : 0.4571        
      Balanced Accuracy : 0.7308        
                                        
       'Positive' Class : 1             
                                        
compute_aupr(subject_level$filter_220_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 220 and skeletonize')

compute_auroc(subject_level$filter_220_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 220 and skeletonize')

confusion_matrix(subject_level, 'filter_220_skeletonized')
Warning in confusionMatrix.default(preds, df$label, positive = "1") :
  Levels are not in the same order for reference and data. Refactoring data to match.
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43 27
         1  0  0
                                          
               Accuracy : 0.6143          
                 95% CI : (0.4903, 0.7283)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.5525          
                                          
                  Kappa : 0               
                                          
 Mcnemar's Test P-Value : 5.624e-07       
                                          
            Sensitivity : 0.0000          
            Specificity : 1.0000          
         Pos Pred Value :    NaN          
         Neg Pred Value : 0.6143          
             Prevalence : 0.3857          
         Detection Rate : 0.0000          
   Detection Prevalence : 0.0000          
      Balanced Accuracy : 0.5000          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 230

compute_aupr(subject_level$filter_230, subject_level$label, 'PR: Segmentations: zero all pixels < 230')

compute_auroc(subject_level$filter_230, subject_level$label, 'ROC: Segmentations: zero all pixels < 230')

confusion_matrix(subject_level, 'filter_230')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 26  6
         1 17 21
                                          
               Accuracy : 0.6714          
                 95% CI : (0.5488, 0.7791)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.19581         
                                          
                  Kappa : 0.3555          
                                          
 Mcnemar's Test P-Value : 0.03706         
                                          
            Sensitivity : 0.7778          
            Specificity : 0.6047          
         Pos Pred Value : 0.5526          
         Neg Pred Value : 0.8125          
             Prevalence : 0.3857          
         Detection Rate : 0.3000          
   Detection Prevalence : 0.5429          
      Balanced Accuracy : 0.6912          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_230_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 230 and binarize')

compute_auroc(subject_level$filter_230_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 230 and binarize')

confusion_matrix(subject_level, 'filter_230_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 10  1
         1 33 26
                                          
               Accuracy : 0.5143          
                 95% CI : (0.3917, 0.6356)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.966           
                                          
                  Kappa : 0.1602          
                                          
 Mcnemar's Test P-Value : 1.058e-07       
                                          
            Sensitivity : 0.9630          
            Specificity : 0.2326          
         Pos Pred Value : 0.4407          
         Neg Pred Value : 0.9091          
             Prevalence : 0.3857          
         Detection Rate : 0.3714          
   Detection Prevalence : 0.8429          
      Balanced Accuracy : 0.5978          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_230_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 230 and skeletonize')

compute_auroc(subject_level$filter_230_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 230 and skeletonize')

confusion_matrix(subject_level, 'filter_230_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 17  2
         1 26 25
                                          
               Accuracy : 0.6             
                 95% CI : (0.4759, 0.7153)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.6466          
                                          
                  Kappa : 0.2757          
                                          
 Mcnemar's Test P-Value : 1.383e-05       
                                          
            Sensitivity : 0.9259          
            Specificity : 0.3953          
         Pos Pred Value : 0.4902          
         Neg Pred Value : 0.8947          
             Prevalence : 0.3857          
         Detection Rate : 0.3571          
   Detection Prevalence : 0.7286          
      Balanced Accuracy : 0.6606          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 240

compute_aupr(subject_level$filter_240, subject_level$label, 'PR: Segmentations: zero all pixels < 240')

compute_auroc(subject_level$filter_240, subject_level$label, 'ROC: Segmentations: zero all pixels < 240')

confusion_matrix(subject_level, 'filter_240')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 23  9
         1 20 18
                                          
               Accuracy : 0.5857          
                 95% CI : (0.4617, 0.7023)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.73214         
                                          
                  Kappa : 0.1873          
                                          
 Mcnemar's Test P-Value : 0.06332         
                                          
            Sensitivity : 0.6667          
            Specificity : 0.5349          
         Pos Pred Value : 0.4737          
         Neg Pred Value : 0.7187          
             Prevalence : 0.3857          
         Detection Rate : 0.2571          
   Detection Prevalence : 0.5429          
      Balanced Accuracy : 0.6008          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_240_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 240 and binarize')

compute_auroc(subject_level$filter_240_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 240 and binarize')

confusion_matrix(subject_level, 'filter_240_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 12  1
         1 31 26
                                          
               Accuracy : 0.5429          
                 95% CI : (0.4194, 0.6626)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.9106          
                                          
                  Kappa : 0.2006          
                                          
 Mcnemar's Test P-Value : 2.951e-07       
                                          
            Sensitivity : 0.9630          
            Specificity : 0.2791          
         Pos Pred Value : 0.4561          
         Neg Pred Value : 0.9231          
             Prevalence : 0.3857          
         Detection Rate : 0.3714          
   Detection Prevalence : 0.8143          
      Balanced Accuracy : 0.6210          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_240_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 240 and skeletonize')

compute_auroc(subject_level$filter_240_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 240 and skeletonize')

confusion_matrix(subject_level, 'filter_240_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 14  2
         1 29 25
                                          
               Accuracy : 0.5571          
                 95% CI : (0.4334, 0.6759)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.8651          
                                          
                  Kappa : 0.2121          
                                          
 Mcnemar's Test P-Value : 3.016e-06       
                                          
            Sensitivity : 0.9259          
            Specificity : 0.3256          
         Pos Pred Value : 0.4630          
         Neg Pred Value : 0.8750          
             Prevalence : 0.3857          
         Detection Rate : 0.3571          
   Detection Prevalence : 0.7714          
      Balanced Accuracy : 0.6258          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 250

compute_aupr(subject_level$filter_250, subject_level$label, 'PR: Segmentations: zero all pixels < 250')

compute_auroc(subject_level$filter_250, subject_level$label, 'ROC: Segmentations: zero all pixels < 250')

confusion_matrix(subject_level, 'filter_250')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0  1  0
         1 42 27
                                          
               Accuracy : 0.4             
                 95% CI : (0.2847, 0.5241)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.9999          
                                          
                  Kappa : 0.018           
                                          
 Mcnemar's Test P-Value : 2.509e-10       
                                          
            Sensitivity : 1.00000         
            Specificity : 0.02326         
         Pos Pred Value : 0.39130         
         Neg Pred Value : 1.00000         
             Prevalence : 0.38571         
         Detection Rate : 0.38571         
   Detection Prevalence : 0.98571         
      Balanced Accuracy : 0.51163         
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_250_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 250 and binarize')

compute_auroc(subject_level$filter_250_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 250 and binarize')

confusion_matrix(subject_level, 'filter_250_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0  1  0
         1 42 27
                                          
               Accuracy : 0.4             
                 95% CI : (0.2847, 0.5241)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.9999          
                                          
                  Kappa : 0.018           
                                          
 Mcnemar's Test P-Value : 2.509e-10       
                                          
            Sensitivity : 1.00000         
            Specificity : 0.02326         
         Pos Pred Value : 0.39130         
         Neg Pred Value : 1.00000         
             Prevalence : 0.38571         
         Detection Rate : 0.38571         
   Detection Prevalence : 0.98571         
      Balanced Accuracy : 0.51163         
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_250_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 250 and skeletonize')

compute_auroc(subject_level$filter_250_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 250 and skeletonize')

confusion_matrix(subject_level, 'filter_250_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0  1  0
         1 42 27
                                          
               Accuracy : 0.4             
                 95% CI : (0.2847, 0.5241)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.9999          
                                          
                  Kappa : 0.018           
                                          
 Mcnemar's Test P-Value : 2.509e-10       
                                          
            Sensitivity : 1.00000         
            Specificity : 0.02326         
         Pos Pred Value : 0.39130         
         Neg Pred Value : 1.00000         
             Prevalence : 0.38571         
         Detection Rate : 0.38571         
   Detection Prevalence : 0.98571         
      Balanced Accuracy : 0.51163         
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 257

compute_aupr(subject_level$filter_257, subject_level$label, 'PR: Segmentations: zero all pixels < 257')

compute_auroc(subject_level$filter_257, subject_level$label, 'ROC: Segmentations: zero all pixels < 257')

confusion_matrix(subject_level, 'filter_257')
Warning in confusionMatrix.default(preds, df$label, positive = "1") :
  Levels are not in the same order for reference and data. Refactoring data to match.
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43 27
         1  0  0
                                          
               Accuracy : 0.6143          
                 95% CI : (0.4903, 0.7283)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.5525          
                                          
                  Kappa : 0               
                                          
 Mcnemar's Test P-Value : 5.624e-07       
                                          
            Sensitivity : 0.0000          
            Specificity : 1.0000          
         Pos Pred Value :    NaN          
         Neg Pred Value : 0.6143          
             Prevalence : 0.3857          
         Detection Rate : 0.0000          
   Detection Prevalence : 0.0000          
      Balanced Accuracy : 0.5000          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_257_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 257 and binarize')

compute_auroc(subject_level$filter_257_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 257 and binarize')

confusion_matrix(subject_level, 'filter_257_binarized')
Warning in confusionMatrix.default(preds, df$label, positive = "1") :
  Levels are not in the same order for reference and data. Refactoring data to match.
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43 27
         1  0  0
                                          
               Accuracy : 0.6143          
                 95% CI : (0.4903, 0.7283)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.5525          
                                          
                  Kappa : 0               
                                          
 Mcnemar's Test P-Value : 5.624e-07       
                                          
            Sensitivity : 0.0000          
            Specificity : 1.0000          
         Pos Pred Value :    NaN          
         Neg Pred Value : 0.6143          
             Prevalence : 0.3857          
         Detection Rate : 0.0000          
   Detection Prevalence : 0.0000          
      Balanced Accuracy : 0.5000          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_257_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 257 and skeletonize')

compute_auroc(subject_level$filter_257_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 257 and skeletonize')

confusion_matrix(subject_level, 'filter_257_skeletonized')
Warning in confusionMatrix.default(preds, df$label, positive = "1") :
  Levels are not in the same order for reference and data. Refactoring data to match.
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43 27
         1  0  0
                                          
               Accuracy : 0.6143          
                 95% CI : (0.4903, 0.7283)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.5525          
                                          
                  Kappa : 0               
                                          
 Mcnemar's Test P-Value : 5.624e-07       
                                          
            Sensitivity : 0.0000          
            Specificity : 1.0000          
         Pos Pred Value :    NaN          
         Neg Pred Value : 0.6143          
             Prevalence : 0.3857          
         Detection Rate : 0.0000          
   Detection Prevalence : 0.0000          
      Balanced Accuracy : 0.5000          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels > 10

compute_aupr(subject_level$filter_10, subject_level$label, 'PR: Segmentations: zero all pixels < 10')

compute_auroc(subject_level$filter_10, subject_level$label, 'ROC: Segmentations: zero all pixels < 10')

confusion_matrix(subject_level, 'filter_10')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 41  8
         1  2 19
                                          
               Accuracy : 0.8571          
                 95% CI : (0.7529, 0.9293)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 7.748e-06       
                                          
                  Kappa : 0.6855          
                                          
 Mcnemar's Test P-Value : 0.1138          
                                          
            Sensitivity : 0.7037          
            Specificity : 0.9535          
         Pos Pred Value : 0.9048          
         Neg Pred Value : 0.8367          
             Prevalence : 0.3857          
         Detection Rate : 0.2714          
   Detection Prevalence : 0.3000          
      Balanced Accuracy : 0.8286          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_10_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 10 and binarize')

compute_auroc(subject_level$filter_10_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 10 and binarize')

confusion_matrix(subject_level, 'filter_10_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 34  0
         1  9 27
                                          
               Accuracy : 0.8714          
                 95% CI : (0.7699, 0.9395)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 1.949e-06       
                                          
                  Kappa : 0.7445          
                                          
 Mcnemar's Test P-Value : 0.007661        
                                          
            Sensitivity : 1.0000          
            Specificity : 0.7907          
         Pos Pred Value : 0.7500          
         Neg Pred Value : 1.0000          
             Prevalence : 0.3857          
         Detection Rate : 0.3857          
   Detection Prevalence : 0.5143          
      Balanced Accuracy : 0.8953          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_10_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 10 and skeletonize')

compute_auroc(subject_level$filter_10_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 10 and skeletonize')

confusion_matrix(subject_level, 'filter_10_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 39  0
         1  4 27
                                          
               Accuracy : 0.9429          
                 95% CI : (0.8601, 0.9842)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 2.41e-10        
                                          
                  Kappa : 0.8826          
                                          
 Mcnemar's Test P-Value : 0.1336          
                                          
            Sensitivity : 1.0000          
            Specificity : 0.9070          
         Pos Pred Value : 0.8710          
         Neg Pred Value : 1.0000          
             Prevalence : 0.3857          
         Detection Rate : 0.3857          
   Detection Prevalence : 0.4429          
      Balanced Accuracy : 0.9535          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 75 and > 150

compute_aupr(subject_level$filter_75, subject_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150')

compute_auroc(subject_level$filter_75, subject_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150')

confusion_matrix(subject_level, 'filter_75')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 36  1
         1  7 26
                                          
               Accuracy : 0.8857          
                 95% CI : (0.7872, 0.9493)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 4.352e-07       
                                          
                  Kappa : 0.7684          
                                          
 Mcnemar's Test P-Value : 0.0771          
                                          
            Sensitivity : 0.9630          
            Specificity : 0.8372          
         Pos Pred Value : 0.7879          
         Neg Pred Value : 0.9730          
             Prevalence : 0.3857          
         Detection Rate : 0.3714          
   Detection Prevalence : 0.4714          
      Balanced Accuracy : 0.9001          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_75_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150 and binarize')

compute_auroc(subject_level$filter_75_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150 and binarize')

confusion_matrix(subject_level, 'filter_75_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 33  0
         1 10 27
                                          
               Accuracy : 0.8571          
                 95% CI : (0.7529, 0.9293)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 7.748e-06       
                                          
                  Kappa : 0.718           
                                          
 Mcnemar's Test P-Value : 0.004427        
                                          
            Sensitivity : 1.0000          
            Specificity : 0.7674          
         Pos Pred Value : 0.7297          
         Neg Pred Value : 1.0000          
             Prevalence : 0.3857          
         Detection Rate : 0.3857          
   Detection Prevalence : 0.5286          
      Balanced Accuracy : 0.8837          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_75_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150 and skeletonize')

compute_auroc(subject_level$filter_75_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150 and skeletonize')

confusion_matrix(subject_level, 'filter_75_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43 13
         1  0 14
                                          
               Accuracy : 0.8143          
                 95% CI : (0.7034, 0.8972)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.0002607       
                                          
                  Kappa : 0.5695          
                                          
 Mcnemar's Test P-Value : 0.0008741       
                                          
            Sensitivity : 0.5185          
            Specificity : 1.0000          
         Pos Pred Value : 1.0000          
         Neg Pred Value : 0.7679          
             Prevalence : 0.3857          
         Detection Rate : 0.2000          
   Detection Prevalence : 0.2000          
      Balanced Accuracy : 0.7593          
                                          
       'Positive' Class : 1               
                                          
LS0tCnRpdGxlOiAnRXZhbHVhdGUgTW9kZWxzJwphdXRob3I6ICdBdXRob3I6IEFhcm9uIFMgQ295bmVyLCBQaEQnCmRhdGU6ICdMYXN0IHVwZGF0ZTogYHIgU3lzLkRhdGUoKWAnCm91dHB1dDoKICAgIGh0bWxfbm90ZWJvb2s6CiAgICAgICAgdG9jOiB5ZXMKICAgICAgICB0b2NfZmxvYXQ6IHllcwogICAgICAgIHRvY19kZXB0aDogMwotLS0KCgojIFNldHVwCmBgYHtyfQpsaWJyYXJ5KHRpZHl2ZXJzZSkKbGlicmFyeSh0b29scykKbGlicmFyeShjYXJldCkKbGlicmFyeShST0NSKQpsaWJyYXJ5KGNhcmV0KQpgYGAKCgpgYGB7cn0KY29tcHV0ZV9hdXJvYyA8LSBmdW5jdGlvbihwcmVkaWN0aW9ucywgbGFiZWxzLCB0aXRsZSkgewogICAgcHJlZCA8LSBwcmVkaWN0aW9uKHByZWRpY3Rpb25zLCBsYWJlbHMpCiAgICBhdXJvYyA8LSBwZXJmb3JtYW5jZShwcmVkLCBtZWFzdXJlID0gJ2F1YycpCiAgICBhdXJvYyA8LSBhdXJvY0B5LnZhbHVlc1tbMV1dCiAgICBwZXJmIDwtIHBlcmZvcm1hbmNlKHByZWQsICd0cHInLCAnZnByJykKICAgIHBsb3QocGVyZiwKICAgICAgICAgbWFpbiA9IHRpdGxlLAogICAgICAgICB4bGltPWMoMCwxKSwKICAgICAgICAgeWxpbT1jKDAsMSksCiAgICAgICAgIGNvbG9yaXplID0gRkFMU0UpCiAgICBhYmxpbmUoYSA9IDAsCiAgICAgICAgICAgYiA9IDEsCiAgICAgICAgICAgY29sID0gJ3JlZCcsCiAgICAgICAgICAgbHR5ID0gMikKICAgIHRleHQoMC44LAogICAgICAgICAwLjI1LAogICAgICAgICBwYXN0ZSgnQVVST0M6Jywgc3ByaW50ZignJS4zZicsIHJvdW5kKGF1cm9jLCAzKSkpKQp9CmBgYAoKCmBgYHtyfQpjb21wdXRlX2F1cHIgPC0gZnVuY3Rpb24ocHJlZGljdGlvbnMsIGxhYmVscywgdGl0bGUpIHsKICAgIG5fYmxhY2sgPC0gbGVuZ3RoKGxhYmVsc1tsYWJlbHMgPT0gMV0pCiAgICBuX3RvdGFsIDwtIGxlbmd0aChsYWJlbHMpCiAgICBwcmVkIDwtIHByZWRpY3Rpb24ocHJlZGljdGlvbnMsIGxhYmVscykKICAgIGF1cHIgPC0gcGVyZm9ybWFuY2UocHJlZCwgbWVhc3VyZSA9ICdhdWNwcicpCiAgICBhdXByIDwtIGF1cHJAeS52YWx1ZXNbWzFdXQogICAgcGVyZiA8LSBwZXJmb3JtYW5jZShwcmVkLCAncHJlYycsICdyZWMnKQogICAgcGxvdChwZXJmLAogICAgICAgICBtYWluID0gdGl0bGUsCiAgICAgICAgIHhsaW09YygwLDEpLAogICAgICAgICB5bGltPWMoMCwxKSwKICAgICAgICAgY29sb3JpemUgPSBGQUxTRSkKICAgIGFibGluZShhID0gbl9ibGFjay9uX3RvdGFsLAogICAgICAgICAgIGIgPSAwLAogICAgICAgICAgIGNvbCA9ICdyZWQnLAogICAgICAgICAgIGx0eSA9IDIpCiAgICB0ZXh0KDAuMiwKICAgICAgICAgMC4yLAogICAgICAgICBwYXN0ZSgnTnVsbCBBVVBSOicsIHNwcmludGYoJyUuM2YnLCByb3VuZChuX2JsYWNrIC8gbl90b3RhbCwgMykpKSkKICAgIHRleHQoMC44LAogICAgICAgICAwLjIsCiAgICAgICAgIHBhc3RlKCdBVVBSOicsIHNwcmludGYoJyUuM2YnLCByb3VuZChhdXByLCAzKSkpKQp9CmBgYAoKCmBgYHtyfQpjb25mdXNpb25fbWF0cml4IDwtIGZ1bmN0aW9uKGRmLCBjb2x1bW4sIHRocmVzaG9sZD0wLjUpIHsKICAgIHByZWRzIDwtIGFzLmZhY3RvcihpZl9lbHNlKGRmW2NvbHVtbl0gPj0gdGhyZXNob2xkLCAxLCAwKSkKICAgIGNvbmZ1c2lvbk1hdHJpeChwcmVkcywgZGYkbGFiZWwsIHBvc2l0aXZlID0gJzEnKQp9CmBgYAoKCgojIExvYWQgZGF0YQpgYGB7cn0KdGVzdF9kYXRhIDwtIHJlYWRfY3N2KCcuL291dC9kYXRhc2V0cy90ZXN0X2RhdGEuY3N2JykgJT4lCiAgICBzZWxlY3Qoc3ViamVjdF9pZCwgaW1hZ2VfaWQpCgoKaW1hZ2VfbGV2ZWwgPC0gcmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvcmV0Y2FtX2ZpbHRlcmVkXzAuY3N2JywgY29sX3R5cGVzID0gY29scygpKSAlPiUKICAgIG11dGF0ZShsYWJlbCA9IGFzLmZhY3RvcihpZl9lbHNlKHN0cl9kZXRlY3QoaW1nX2xvYywgJ2JsYWNrJyksIDEsIDApKSwKICAgICAgICAgICBpbWFnZV9pZCA9IGFzLm51bWVyaWMoZmlsZV9wYXRoX3NhbnNfZXh0KGJhc2VuYW1lKGltZ19sb2MpKSkpICU+JQogICAgc2VsZWN0KGltZ19sb2MsIGltYWdlX2lkLCBsYWJlbCwgcmV0Y2FtID0gcHJvYmFiaWxpdHkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9yZXRjYW1fZmlsdGVyZWRfMF9yYW5kb20uY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgcmV0Y2FtX3JhbmRvbSA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICAKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8wX3JhbmRvbS5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMF9yYW5kb20gPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzAuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzAgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzBfYmluYXJpemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8wX2JpbmFyaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMF9za2VsZXRvbml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzBfc2tlbGV0b25pemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIAogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzUwLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl81MCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfNTBfYmluYXJpemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl81MF9iaW5hcml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzUwX3NrZWxldG9uaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfNTBfc2tlbGV0b25pemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMTAwLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8xMDAgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzEwMF9iaW5hcml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzEwMF9iaW5hcml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzEwMF9za2VsZXRvbml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzEwMF9za2VsZXRvbml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQoKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8xNTAuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzE1MCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMTUwX2JpbmFyaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMTUwX2JpbmFyaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMTUwX3NrZWxldG9uaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMTUwX3NrZWxldG9uaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCgogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzIwMC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjAwID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yMDBfYmluYXJpemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yMDBfYmluYXJpemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yMDBfc2tlbGV0b25pemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yMDBfc2tlbGV0b25pemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjEwLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yMTAgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzIxMF9iaW5hcml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzIxMF9iaW5hcml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzIxMF9za2VsZXRvbml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzIxMF9za2VsZXRvbml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQoKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yMjAuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzIyMCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjIwX2JpbmFyaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjIwX2JpbmFyaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjIwX3NrZWxldG9uaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjIwX3NrZWxldG9uaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCgogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzIzMC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjMwID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yMzBfYmluYXJpemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yMzBfYmluYXJpemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yMzBfc2tlbGV0b25pemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yMzBfc2tlbGV0b25pemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjQwLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yNDAgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzI0MF9iaW5hcml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzI0MF9iaW5hcml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzI0MF9za2VsZXRvbml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzI0MF9za2VsZXRvbml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQoKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yNTAuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzI1MCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjUwX2JpbmFyaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjUwX2JpbmFyaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjUwX3NrZWxldG9uaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjUwX3NrZWxldG9uaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCgogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzI1Ny5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjU3ID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yNTdfYmluYXJpemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yNTdfYmluYXJpemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yNTdfc2tlbGV0b25pemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yNTdfc2tlbGV0b25pemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMF8xMC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMTAgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzBfMTBfYmluYXJpemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8xMF9iaW5hcml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzBfMTBfc2tlbGV0b25pemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8xMF9za2VsZXRvbml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQoKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF83NV8xNTAuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzc1ID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF83NV8xNTBfYmluYXJpemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl83NV9iaW5hcml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzc1XzE1MF9za2VsZXRvbml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzc1X3NrZWxldG9uaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICAKICAgIGlubmVyX2pvaW4odGVzdF9kYXRhLCBieSA9ICdpbWFnZV9pZCcpICU+JQogICAgc2VsZWN0KHN1YmplY3RfaWQsIGV2ZXJ5dGhpbmcoKSwgLWltZ19sb2MpCgoKCnN1YmplY3RfbGV2ZWwgPC0gaW1hZ2VfbGV2ZWwgJT4lCiAgICBncm91cF9ieShzdWJqZWN0X2lkKSAlPiUKICAgIG11dGF0ZShhY3Jvc3MoYygtaW1hZ2VfaWQsIC1sYWJlbCksIG1lZGlhbikpICU+JQogICAgdW5ncm91cCgpICU+JQogICAgZGlzdGluY3Qoc3ViamVjdF9pZCwgLmtlZXBfYWxsID0gVFJVRSkKYGBgCgoKIyBJbWFnZS1sZXZlbCBBbmFseXNpcwoKIyMjIFJldENhbSBJbWFnZXMKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkcmV0Y2FtLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBSYXcgUmV0Q2FtIEltYWdlcycpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkcmV0Y2FtLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogUmF3IFJldENhbSBJbWFnZXMnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAncmV0Y2FtJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJHJldGNhbV9yYW5kb20sIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFJhdyBSZXRDYW0gSW1hZ2VzIC0gU2h1ZmZsZWQgTGFiZWxzJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRyZXRjYW1fcmFuZG9tLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogUmF3IFJldENhbSBJbWFnZXMgLSBTaHVmZmxlZCBMYWJlbHMnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAncmV0Y2FtX3JhbmRvbScpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzAsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8wJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8wX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8wX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8wX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMF9za2VsZXRvbml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzBfcmFuZG9tLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwIC0gU2h1ZmZsZWQgTGFiZWxzJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMF9yYW5kb20sIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwIC0gU2h1ZmZsZWQgTGFiZWxzJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8wX3JhbmRvbScpCmBgYAoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA1MAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfNTAsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfNTAsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA1MCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfNTAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzUwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNTAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfNTBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNTAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8wX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfNTBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA1MCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl81MF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA1MCBhbmQgc2tlbGV0b25pemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzUwX3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8xMDAsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwMCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwMCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMTAwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8xMDBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMDAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMTAwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwMCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzEwMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwMCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8xMDBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMTAwX3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8xNTAsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDE1MCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzE1MCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDE1MCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMTUwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8xNTBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxNTAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMTUwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDE1MCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzE1MF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzE1MF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDE1MCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8xNTBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMTUwX3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yMDAsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIwMCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIwMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIwMCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjAwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yMDBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMDAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjAwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIwMCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzIwMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIwMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIwMCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yMDBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjAwX3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yMTAsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIxMCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIxMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIxMCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjEwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yMTBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMTAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjEwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIxMCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzIxMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIxMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIxMCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yMTBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjEwX3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yMjAsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIyMCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIyMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIyMCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjIwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yMjBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMjAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjIwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIyMCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzIyMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIyMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIyMCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yMjBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjIwX3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yMzAsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIzMCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIzMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIzMCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjMwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yMzBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMzAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjMwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIzMCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzIzMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIzMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIzMCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yMzBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjMwX3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yNDAsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI0MCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI0MCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI0MCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjQwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yNDBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNDAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjQwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI0MCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzI0MF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI0MF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI0MCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yNDBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjQwX3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yNTAsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1MCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI1MCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1MCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjUwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yNTBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjUwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1MCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzI1MF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI1MF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1MCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yNTBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjUwX3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3CgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yNTcsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1NycpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI1NywgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1NycpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjU3JykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yNTdfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTcgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjU3X2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1NyBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzI1N19iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI1N19za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1NyBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yNTdfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3IGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjU3X3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzID4gMTAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzEwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8xMF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMTBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8xMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMTBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA3NSBhbmQgPiAxNTAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzc1LCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA3NSBhbmQgPiAxNTAnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl83NSwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDc1IGFuZCA+IDE1MCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfNzUnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzc1X2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNzUgYW5kID4gMTUwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzc1X2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDc1IGFuZCA+IDE1MCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzc1X2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfNzVfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA3NSBhbmQgPiAxNTAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfNzVfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNzUgYW5kID4gMTUwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfNzVfc2tlbGV0b25pemVkJykKYGBgCgoKCgoKCgoKCgoKIyBTdWJqZWN0LWxldmVsIEFuYWx5c2lzCgojIyMgUmV0Q2FtIEltYWdlcwoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJHJldGNhbSwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBSYXcgUmV0Q2FtIEltYWdlcycpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRyZXRjYW0sIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFJhdyBSZXRDYW0gSW1hZ2VzJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAncmV0Y2FtJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkcmV0Y2FtX3JhbmRvbSwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBSYXcgUmV0Q2FtIEltYWdlcyAtIFNodWZmbGVkIExhYmVscycpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRyZXRjYW1fcmFuZG9tLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBSYXcgUmV0Q2FtIEltYWdlcyAtIFNodWZmbGVkIExhYmVscycpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ3JldGNhbV9yYW5kb20nKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8wLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8wX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8wX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzBfc2tlbGV0b25pemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzBfcmFuZG9tLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAgLSBTaHVmZmxlZCBMYWJlbHMnKQpjb21wdXRlX2F1cm9jKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzBfcmFuZG9tLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwIC0gU2h1ZmZsZWQgTGFiZWxzJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzBfcmFuZG9tJykKYGBgCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzUwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl81MCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNTAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfNTAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfNTBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfNTBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA1MCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfNTBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfNTBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA1MCBhbmQgc2tlbGV0b25pemUnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfNTBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMDAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTAwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwMCcpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTAwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMDAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMTAwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzEwMF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTAwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8xMDBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzEwMF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTAwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8xMDBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxNTAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTUwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDE1MCcpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTUwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxNTAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMTUwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzE1MF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTUwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8xNTBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzE1MF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTUwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8xNTBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMDAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjAwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIwMCcpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjAwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMDAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMjAwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzIwMF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjAwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMDBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzIwMF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjAwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMDBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMTAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjEwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIxMCcpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjEwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMTAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMjEwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzIxMF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjEwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMTBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzIxMF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjEwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMTBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMjAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjIwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIyMCcpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjIwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMjAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMjIwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzIyMF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjIwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMjBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzIyMF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjIwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMjBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMzAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjMwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIzMCcpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjMwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMzAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMjMwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzIzMF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjMwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMzBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzIzMF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjMwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMzBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNDAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjQwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI0MCcpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjQwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNDAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMjQwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzI0MF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjQwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yNDBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzI0MF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjQwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yNDBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjUwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1MCcpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjUwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMjUwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzI1MF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjUwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yNTBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzI1MF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjUwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yNTBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTcKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjU3LCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1NycpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjU3LCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTcnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMjU3JykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzI1N19iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3IGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjU3X2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3IGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yNTdfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzI1N19za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3IGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjU3X3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3IGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yNTdfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPiAxMAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8xMCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMCcpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzEwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzEwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzEwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzEwX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8xMF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8xMF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8xMF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDc1IGFuZCA+IDE1MAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl83NSwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA3NSBhbmQgPiAxNTAnKQpjb21wdXRlX2F1cm9jKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzc1LCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA3NSBhbmQgPiAxNTAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfNzUnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfNzVfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDc1IGFuZCA+IDE1MCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzc1X2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNzUgYW5kID4gMTUwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl83NV9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfNzVfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDc1IGFuZCA+IDE1MCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzc1X3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNzUgYW5kID4gMTUwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl83NV9za2VsZXRvbml6ZWQnKQpgYGAK